'use strict'

const { TaiSpider } = require('tai-spider');
const { formatDate } = require('../utils/str-helper');

class CneosnewsSpider extends TaiSpider {

    constructor(options = {}) {
        super(options);
        this.name = 'cneos-news';
        this.debug = true;
        this.start_urls = [
            'https://cneos.jpl.nasa.gov/news/news.json'
        ];
    }

    *parse(response) {
        let json = response.getJSON();
        // console.log(json.news.length)
        for (let ele of json.news) {
            yield response.follow(ele.url, this.parseArticle);
        }
    }

    *parseArticle(response, spider) {
        let data = response.extract();
        let date = formatDate(data.date)
        data.timestamp = date.valueOf();
        data.date = date.format();

        yield Object.assign({
            index_name: 'tai-article',
        }, data);
    }
}

module.exports = CneosnewsSpider;